# -*- coding: iso-8859-2 -*-

from authorshipanalyzer.file import ResourceReader
from authorshipanalyzer.file import Path
from authorshipanalyzer.slot import Slot
from authorshipanalyzer.textutils import Splitter
from authorshipanalyzer.utils import getKey

import math
import os

def sortFileList(paths):
    '''
    Sort and remove dupliactes in the file with list of words
    '''
    
    reader = ResourceReader()
    for path in paths:
        for file in os.listdir(path):
            if file == ".svn":
                continue
            
            fullPath = path + os.sep + file
            words = reader.readWordList(fullPath)
            words = list(set(words))
            words.sort()
            f = open(fullPath, "w")
            for word in words:
                f.write(word + "\n")
            f.close()
        
def extractKeyWords(messages):
    '''
    Extract key words for categories. Assign weights.
    '''        
    splitter = Splitter()
    
    keyWordsDirs = [ "categories", "subjects" ]
    categoriesWords = {}
        
    reader = ResourceReader()
    for dir in keyWordsDirs:
        categoryNames = reader.getMechanizmFilesDict(dir)
        for category in categoryNames.keys():
            categoryWords = reader.readWordList(categoryNames[category])
            categoriesWords[category] = categoryWords
    
    # initially all weights are set to zeros, keys are all words from all categories
    weights = {}        
    for key, value in categoriesWords.items():
        for word in value:
            weights[word] = 0.0
    
    i = 0 
    N = 0.0
    for message in messages:
        N += 1.0
        
        wordFrequency = splitter.getWordFrequencyDict(message.getText())
        for word, freq in wordFrequency.items():
            word = getKey(weights, word)
            counted = [] 
            if word != None and word not in counted:
                weights[word] += 1.0
                counted.append(word)
                
        i += 1
        if i == 100 or N == len(messages):
            i = 0
            print str(N)
#            for word, weight in weights.items():
#                # count weight otherwise log(df/N)
#                if weight != 0.0:
#                    print word + " " + str(N) + " / " + str(weight) + " = " + str(math.log(N / weight))
#                    weights[word] = math.log(N / weight)   
            
            for key, words in categoriesWords.items():
#                sum = 0.0
#                for word in words:
#                    if weights.has_key(word):
#                        sum += weights[word] * weights[word]
#                
#                sum = math.sqrt(sum)
                        
#                if sum != 0.0:
                print "== " + key
                file = open("/tmp/" + key + ".txt", "w")
                for word in words:
                    if weights.has_key(word) and weights[word] > 0.0:
#                        print word + " " + str(weights[word]) + " " + str(math.log(N / weights[word]))
                        file.write(word + " " + str(math.log(N / weights[word])) + "\n")
                    else:
#                        print word + " 0.0"
                        file.write(word + " 0.0\n")
                file.close()   


     
        
            
#    for key, words in categoriesWords.items():
#        print "--------> cat: " + key
#        file = open("/tmp/" + key + ".txt", "w")
#        for word in words:
#            if weights.has_key(word):
#                print word + " " + str(weights[word])
#                file.write(word + " " + str(weights[word]) + "\n")
#            else:
#                print word + " 0.0"
#                file.write(word + " 0.0\n")
#        file.close()
        
if __name__ == "__main__":
#    paths = [Path.getCategoriesDir(), Path.getSubjectsDir()]
#    sortFileList(paths)

    import datetime
    import os
    from time import time
    
    from authorshipanalyzer.message import Message
    from authorshipanalyzer.message import MessageReader
    from authorshipanalyzer.file import Path
    from authorshipanalyzer.textmodifier import MessagePreprocessor
    from authorshipanalyzer.slot import Slot
    from authorshipanalyzer.slot import SlotManager

    fromDate = datetime.date(1996, 1, 1)
    toDate = datetime.date(2009, 2, 3)
    
    start = time()
    mr = MessageReader(Path.getMainDir() + os.sep + "info1")
    messages = mr.readMessages(fromDate, toDate)
    end = time()
    print "Retrieving " + str(len(messages)) + " messages: " + str(end - start) + " seconds"


    extractKeyWords(messages)